import sys, os, time, Numeric;

user_name = "svilen";
remotescript_dir = "~/exp/source/build/micaz/";
result_machine = "dbfileserv.cis.upenn.edu";
result_dir = "/projectsloc/"+user_name+"/vldb2008_1/results";
result_filename = "";

expid_file = "exp_id.txt";
explog_file = "exp_log.txt";
systemlog_file = "system_log.txt";

proc_index = 0; 
proc_count = 1; 
proc_share = 1;
proc_label = 0;
method = "";
exp_id = "0";

param_count = 29;
filter_param = (0, 1, 4, 7, 16);
filtered_paramcount = param_count-len(filter_param);


def get_timestamp():
        return time.strftime("%Y/%m/%d %H:%M:%S", time.localtime());


def read_config(file_name):
        dictionary = {};
        try:
                f = open(file_name, "r");
                ll = f.readlines();
                for l in ll:
                        l1 = (l.split('\n')[0]).split('=');
                        dictionary[l1[0]] = l1[1];
                f.close();
        except:
                dictionary = {};

        return dictionary;


def write_config(file_name, dictionary):
        f = open(file_name, "w");
        for k, v in dictionary.iteritems():
                f.write(k+"="+v+"\n");
        f.close();


def system_cmd(cmd):
        f = open(systemlog_file, "a");
        f.write(get_timestamp()+": "+cmd+"\n");
        f.close();

	os.system("nice "+cmd);


def in_range(part_index, part_count):
	return (proc_count*part_index >= proc_index*part_count) & (proc_count*part_index < (proc_index+proc_share)*part_count);


def gen_stats(dir):
	stats = Numeric.zeros((filtered_paramcount ,3), 'l');

	for i in range(0, int(cf['node_count'])):
		f_name = dir+"stats"+str(i)+".txt";
		try:
			f = open(f_name, "r");
		except:
			print("Cannot open "+f_name);
			return;

		ll = f.readlines();
		if (len(ll) < 1):
			print("File "+f_name+" is empty!");
			return;

		l = ll[0].split('\n')[0].split(' ');
		k = 0;
		for j in range(0, param_count):
			if (j in filter_param):
				continue;
			stats[k][0] = stats[k][0] + long(l[j]);
			if (long(l[j]) > stats[k][1]):
				stats[k][1] = long(l[j]);
				stats[k][2] = i;
			k = k+1;

		f.close();

	system_cmd("rm "+dir+"stats*.txt");
        try:
                f = open(dir+"stats_aggr.txt", "w");
        except:
                print("Cannot create stats_aggr.txt");
                return;
        for j in range(0, filtered_paramcount):
                f.write(str(stats[j][0])+" "+str(stats[j][1])+" "+str(stats[j][2])+"\n");
        f.close();


def analyze_stats():
        global routedir_list;
        for s in routedir_list:
                gen_stats(s+'/');


def update_progress(part_index, part_count):
	exp_info["percent_complete"] = str(100*(proc_count*part_index-proc_index*part_count)/(proc_share*part_count));
	write_config(expid_file, exp_info);

        
def exp_log(s):
	s = get_timestamp()+": "+s;
	print(s);

	f_explog = open(explog_file, "a");
	f_explog.write(s+"\n");
	f_explog.close();


def archive_result(dir_mask):
	analyze_stats();
	system_cmd("tar -cf "+result_filename+" -z "+dir_mask);
	system_cmd("rm -r "+dir_mask);	
	

def exp_treebuild():
	test_topologies = ('100-1-sq', '100-1-sq-broken-10-1', '100-1-sq-broken-21-1', '100-n-1-1', '100-n-1-2', '100-n-1-3', '100-n-2-1', '100-n-2-2', '100-n-2-3', '100-n-3-1', '100-n-3-2', '100-n-3-3', '100-n-4-1', '100-n-4-2', '100-n-4-3', '250-n-1', '200-n-1', '50-n-1', '55-intel');
	test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10');

	part_count = len(test_topologies)*len(test_tc);
	part_index = 0;
	for top in test_topologies:
		for tc in test_tc:
			if (in_range(part_index, part_count)):
				exp_log("exp_treebuild: top: "+str(top)+", tc: "+str(tc));
				build_trees((top,), (tc,));
				update_progress(part_index, part_count);
			part_index = part_index+1;

	archive_result("trees-*");
	

def exp_allpairroute():
	test_topologies = ('100-n-1-1', '100-n-1-2', '100-n-1-3', '100-n-2-1', '100-n-2-2', '100-n-2-3', '100-n-3-1', '100-n-3-2', '100-n-3-3', '100-n-4-1', '100-n-4-2', '100-n-4-3', '100-1-sq', '100-1-sq-broken-10-1', '100-1-sq-broken-21-1');
	test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
	test_treesenabled = ('1', '2', '3');

	part_count = 0;
	for top in test_topologies:
		N = int(top.split('-')[0]);
		part_count = part_count + N*N/10;
	part_count = part_count*len(test_tc)*len(test_treesenabled);	

	part_index = 0;
	for top in test_topologies:
		N = int(top.split('-')[0]);
                batch_count = N*N/10;

		for tc in test_tc:
			for te in test_treesenabled:
				for bi in range(0, batch_count):
					if (in_range(part_index, part_count)):
        		                        exp_log("exp_allpairroute: top: "+str(top)+", tc: "+str(tc)+", te:"+str(te)+", bi:"+str(bi));
	                	                test_allpairroute((top,), (tc,), (te,), '0', '0', '0 0', 10, bi, bi);
                                		update_progress(part_index, part_count);
					part_index = part_index+1;

	archive_result("allpair_route*");


def exp_allpairrouteght():
        test_topologies = ('100-n-1-1', '100-n-1-2', '100-n-1-3', '100-n-2-1', '100-n-2-2', '100-n-2-3', '100-n-3-1', '100-n-3-2', '100-n-3-3', '100-n-4-1', '100-n-4-2', '100-n-4-3', '100-1-sq', '100-1-sq-broken-10-1', '100-1-sq-broken-21-1');
	dim_random = ('60 60', '90 90', '80 80', '100 100');
        test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');

        part_count = 0;
        for top in test_topologies:
                N = int(top.split('-')[0]);
                part_count = part_count + N*N/10;
        part_count = part_count*len(test_tc);

        part_index = 0;
        for top in test_topologies:
		top_split = top.split('-');
                N = int(top_split[0]);
                batch_count = N*N/10;

		if (top_split[1] == 'n'):
			dim = dim_random[int(top_split[2])-1];
		elif (top == '100-1-sq'):
			dim = '90 90';
		elif (top == '100-1-sq-broken-10-1'):
			dim = '100 90';
		elif (top == '100-1-sq-broken-21-1'):
			dim = '100 100';
		else:
			print("Unknown topology dimenstion for "+top);
			sys.exit();

                for tc in test_tc:
                	for bi in range(0, batch_count):
                        	if (in_range(part_index, part_count)):
                                	exp_log("exp_allpairrouteght: top: "+str(top)+", tc: "+str(tc)+", dim:"+str(dim)+", bi:"+str(bi));
                                        test_allpairroute((top,), (tc,), ('3',), '0', '1', dim, 10, bi, bi);
                                        update_progress(part_index, part_count);
                                part_index = part_index+1;

        archive_result("allpair_route*");


def exp_joinvar():
#	test_topologies = ('100-n-3-1', '100-n-3-2', '100-n-3-3');
	test_topologies = ('100-n-3-1',);
	test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
	joinpointsel_config = (('10','1'), ('6','2'), ('2','2'), ('2','6'), ('1','10'));
	realsel_config = joinpointsel_config;
#	join_sel = ('5', '10', '20');
	join_sel = ('5',);
#	window_size = ('1', '3');
	window_size = ('3',);
	sampling_cycles = (100,);
	sampling_time = (80000,);
#	test_methods = (5, 6);
	test_methods = (5,);
#	query_types = ('0', '2', '5', '6');
	query_types = ('0',);
	est_sel = '1';

	part_count = len(test_topologies)*len(test_tc)*len(joinpointsel_config)*len(realsel_config)*len(join_sel)*len(window_size)*len(test_methods)*len(query_types)*len(sampling_cycles)*len(sampling_time);
	part_index = 0;
	for top in test_topologies:
		for tci in test_tc:
			for jps in joinpointsel_config:
				for rs in realsel_config:
					for js in join_sel:
						for m in test_methods:							
							for qt in query_types:
								for ws in window_size:
									for st in sampling_time:
										for sc in sampling_cycles:
											if (in_range(part_index, part_count)):
												exp_log("exp_joinvar: top: "+str(top)+", tci: "+str(tci)+", rs: "+str(rs)+", jps: "+str(jps)+", js: "+str(js)+", ws: "+str(ws)+", m:"+str(m)+", qt:"+str(qt)+", ws:"+str(ws)+", st:"+str(st)+", sc:"+str(sc));
												test_joinvar((top,), (tci,), (rs,), (jps,), (js,), (m,), qt, ws, st, sc, est_sel);
												update_progress(part_index, part_count);
											part_index = part_index+1;
	
	archive_result("join*");


def exp_joinmethods():
#	test_topologies = ('100-n-3-1', '100-n-3-2', '100-n-3-3');
        test_topologies = ('100-n-3-1',);
	test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
	srcsel_config = (('10','1'), ('6','2'), ('2','2'), ('2','6'), ('1','10'));
#	join_sel = ('5', '10', '20');
	join_sel = ('20',);
#	test_methods = (0, 1, 2, 3, 4, 5, 6);
	test_methods = (8,);
	window_size = ('3',);
        sampling_cycles = (100,);
        sampling_time = (10000,);
	query_types = ('2',);

        part_count = len(test_topologies)*len(test_tc)*len(srcsel_config)*len(join_sel)*len(test_methods)*len(window_size)*len(query_types)*len(sampling_time)*len(sampling_cycles);
        part_index = 0;
        for top in test_topologies:
		for tci in test_tc:
	                for srcsel in srcsel_config:
				for js in join_sel:
					for m in test_methods:		
						for qt in query_types:
							for ws in window_size:
								for st in sampling_time:
									for sc in sampling_cycles:
						        	                if (in_range(part_index, part_count)):
											exp_log("exp_joinmethods: top: "+str(top)+", tci: "+str(tci)+", srcsel: "+str(srcsel)+", js: "+str(js)+", m: "+str(m)+", qt:"+str(qt)+", ws:"+str(ws)+", st:"+str(st)+", sc:"+str(sc));
			                			        	        test_joinmethods((top,), (tci,), (srcsel,), (js,), (m,), qt, ws, st, sc);
											update_progress(part_index, part_count);
							                        part_index = part_index+1;

        archive_result("join*");


def exp_joinmethods_learn():
	test_topologies = ('100-n-3-1',);
	test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
        srcsel_1 = ('10','1');
        srcsel_2 = ('1','10');
        jsel_1 = '20';
        jsel_2 = '5';
        param_config = (
          ((srcsel_1, srcsel_1, srcsel_2), (jsel_1, jsel_1, jsel_2), ('0', '0'), '0'),
          ((srcsel_2, srcsel_1, srcsel_2), (jsel_2, jsel_1, jsel_2), ('0', '0'), '0'),
          ((srcsel_1, srcsel_1, srcsel_2), (jsel_1, jsel_1, jsel_2), ('0', '0'), '1'),
          ((srcsel_2, srcsel_1, srcsel_2), (jsel_2, jsel_1, jsel_2), ('0', '0'), '1'),
          ((srcsel_1, srcsel_1, srcsel_2), (jsel_1, jsel_1, jsel_2), ('0', '1'), '0'),
          ((srcsel_2, srcsel_1, srcsel_2), (jsel_2, jsel_1, jsel_2), ('0', '1'), '0'),
          ((srcsel_1, srcsel_1, srcsel_2), (jsel_1, jsel_1, jsel_2), ('1', '1'), '0'),
          ((srcsel_1, srcsel_1, srcsel_2), (jsel_1, jsel_1, jsel_2), ('0', '1'), '1'),
          ((srcsel_2, srcsel_1, srcsel_2), (jsel_2, jsel_1, jsel_2), ('0', '1'), '1')
        );
	window_size = ('1',);
        sampling_cycles = (100,);
        sampling_time = (80000,);
	query_types = ('5',);


        part_count = len(test_topologies)*len(test_tc)*len(param_config)*len(window_size)*len(query_types)*len(sampling_time)*len(sampling_cycles);
        part_index = 0;
        for top in test_topologies:
		for tci in test_tc:
	                for pc in param_config:                                
                        	for qt in query_types:
                                	for ws in window_size:
                                        	for st in sampling_time:
                                                	for sc in sampling_cycles:
                                                        	if (in_range(part_index, part_count)):
                                                                	exp_log("exp_joinmethods_learn: top: "+str(top)+", tci: "+str(tci)+", param: "+str(pc)+", qt:"+str(qt)+", ws:"+str(ws)+", st:"+str(st)+", sc:"+str(sc));
                                                                        test_joinmethods_learn((top,), (tci,), pc[0], pc[1], qt, ws, st, sc, pc[2], pc[3]);
                                                                        update_progress(part_index, part_count);
                                                                part_index = part_index+1;

        archive_result("join*");


def exp_ghtjoin():
        test_topologies = ('100-n-3-1', '100-n-3-2', '100-n-3-3');
	top_dim = '60 60';
        test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
        srcsel_config = (('10','1'), ('6','2'), ('2','2'), ('2','6'), ('1','10'));
        join_sel = ('5', '10', '20');
        window_size = ('1','3');
        sampling_cycles = (100,);
        sampling_time = (10000,);
        query_types = ('2','5','6');

        part_count = len(test_topologies)*len(test_tc)*len(srcsel_config)*len(join_sel)*len(window_size)*len(query_types)*len(sampling_time)*len(sampling_cycles);
        part_index = 0;
        for top in test_topologies:
                for tci in test_tc:
                        for srcsel in srcsel_config:
                                for js in join_sel:                                        
                                	for qt in query_types:
                                        	for ws in window_size:
                                                	for st in sampling_time:
                                                        	for sc in sampling_cycles:
                                                                	if (in_range(part_index, part_count)):
                                                                                exp_log("exp_ghtjoin: top: "+str(top)+", tci: "+str(tci)+", srcsel: "+str(srcsel)+", js: "+str(js)+", qt:"+str(qt)+", ws:"+str(ws)+", st:"+str(st)+", sc:"+str(sc));
                                                                                test_ghtjoinvar((top,), (tci,), (srcsel,), (js,), top_dim, qt, ws, st, sc);
                                                                                update_progress(part_index, part_count);
                                                                        part_index = part_index+1;
	
	archive_result("join*");


def exp_joinintel():
	test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
#	test_methods = (0, 1, 2, 3);
	test_methods = (1,);
	est_sel = '1';
	
	part_count = len(test_tc)*len(test_methods);
	part_index = 0;
	for tci in test_tc:
		for m in test_methods:
			if (in_range(part_index, part_count)):
				exp_log("exp_joinintel: tci:"+str(tci)+", m:"+str(m));
				route_intel((tci,), (m,), est_sel);
				update_progress(part_index, part_count);
			part_index = part_index+1;

	archive_result("join*");


def exp_joinintelstress():
        test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
        test_methods = (0, 1, 2, 3);
#	test_methods = (3,);
	sampling_cycles = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 15, 20, 25, 30, 40, 50, 60, 75, 100, 125, 150, 200, 300);

        part_count = len(test_tc)*len(test_methods)*len(sampling_cycles);
        part_index = 0;
        for tci in test_tc:
                for m in test_methods:
			for sc in sampling_cycles:
	                        if (in_range(part_index, part_count)):
        	                        exp_log("exp_joinintelstress: tci:"+str(tci)+", m:"+str(m)+", sc:"+str(sc));
                	                route_intelstress((tci,), (m,), (sc,));
                        	        update_progress(part_index, part_count);
	                        part_index = part_index+1;

        archive_result("join*");


def exp_stresstest():
        test_topologies = ('100-n-3-1', '100-n-3-2', '100-n-3-3');
	top_dim = '60 60';
        test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
        srcsel_config = (('10','1'), ('1','10'), ('2','2'), ('2','6'), ('6','2'));
        join_sel = ('20',);
        test_methods = (0, 1, 5, 7);
        window_size = ('3',);
        sampling_cycles = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 15, 20, 25, 30, 40, 50, 60, 75, 100, 125, 150, 200, 300);
        query_types = ('2',);
	
        part_count = len(test_topologies)*len(test_tc)*len(srcsel_config)*len(join_sel)*len(test_methods)*len(window_size)*len(sampling_cycles)*len(query_types);
        part_index = 0;
        for top in test_topologies:
                for tci in test_tc:
                        for srcsel in srcsel_config:
                                for js in join_sel:
					for m in test_methods:
	                                        for qt in query_types:
        	                                        for ws in window_size:
                                                                for sc in sampling_cycles:
                                                                        if (in_range(part_index, part_count)):
                                                                                exp_log("exp_stresstest: top: "+str(top)+", tci: "+str(tci)+", srcsel: "+str(srcsel)+", js:"+str(js)+", m:"+str(m)+", qt:"+str(qt)+", ws:"+str(ws)+", sc:"+str(sc));
										test_joinmethodsstress((top,), (tci,), (srcsel,), (js,), (m,), (sc,), qt, ws);
                                                                                update_progress(part_index, part_count);
                                                                        part_index = part_index+1;

        archive_result("join*");


def exp_repairest():
        test_topologies = ('100-n-3-1', '100-n-3-2', '100-n-3-3');
        test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
        srcsel_config = (('2','2'),);
        join_sel = ('20',);
        test_methods = (2, 5);
        window_size = ('3',);
        query_types = ('2',);
	fail_nodes = {3:10, 4:12, 5:14, 6:16, 7:18, 8:20, 10:22, 11:24, 13:26, 15:28, 16:30, 20:32, 21:34, 22:36, 23:38};
	repair_level = ('0', '1', '2', '3');

	part_count = len(test_topologies)*len(test_tc)*len(srcsel_config)*len(join_sel)*len(test_methods)*len(window_size)*len(query_types)*len(repair_level);
	part_index = 0;
        for top in test_topologies:
                for tci in test_tc:
                        for srcsel in srcsel_config:
                                for js in join_sel:
                                        for m in test_methods:
                                                for qt in query_types:
                                                        for rl in repair_level:
                                                                for ws in window_size:
                                                        	        if (in_range(part_index, part_count)):
                                                                                exp_log("exp_repairest: top: "+str(top)+", tci: "+str(tci)+", srcsel: "+str(srcsel)+", js:"+str(js)+", m:"+str(m)+", qt:"+str(qt)+", rl:"+str(rl)+", ws:"+str(ws));
                                                                                test_repairest((top,), (tci,), (srcsel,), (js,), (m,), len(fail_nodes), fail_nodes, qt, ws, rl);
                                                                                update_progress(part_index, part_count);
                                                                        part_index = part_index+1;

        archive_result("join*");


def exp_repairsampling():
        test_topologies = ('100-n-3-1',);
        test_tc = ('1', '2', '3', '4', '5', '6', '7', '8', '9');
        srcsel_config = (('2','2'),);
        join_sel = ('5', '10', '20');
        test_methods = (5,);
        window_size = ('3',);
        repair_level = '2';

        part_count = len(test_topologies)*len(test_tc)*len(srcsel_config)*len(join_sel)*len(test_methods)*len(window_size);
        part_index = 0;
        for top in test_topologies:
                for tci in test_tc:
                        for srcsel in srcsel_config:
				for js in join_sel:
	                        	for m in test_methods:
        	                                for ws in window_size:
                	                                if (in_range(part_index, part_count)):
                        	                                exp_log("exp_repairsampling: top: "+str(top)+", tci: "+str(tci)+", srcsel:"+str(srcsel)+", js:"+str(js)+", m:"+str(m)+", ws:"+str(ws));
	                                                        test_repairsampling((top,), (tci,), (srcsel,), (js,), (m,), ws, repair_level);
        	                                                update_progress(part_index, part_count);
                	                                part_index = part_index+1;

        archive_result("join*");


#---------------------------------------MAIN----------------------------------
os.system("screen -D");
exp_info = read_config(expid_file);

try:
	proc_index = int(exp_info["proc_index"]);
	proc_count = int(exp_info["proc_count"]);
	proc_share = int(exp_info["proc_share"]);
	proc_label = int(exp_info["proc_label"]);
	method = exp_info["param1"];
	exp_id = exp_info["exp_id"];
except:
	print("Missing needed parameters!");
	sys.exit();

if ((proc_count < 1) | (proc_share < 1) | (proc_index < 0) | (proc_index >= proc_count) | (proc_share > proc_count)):
	print("Invalid process parameters!");
	sys.exit();

system_cmd("cp "+remotescript_dir+"_TOSSIMmodule.so .");
system_cmd("cp "+remotescript_dir+"TOSSIM.py .");
system_cmd("cp "+remotescript_dir+"config.py .");
system_cmd("cp "+remotescript_dir+"test_script.py .");
from config import *;

result_filename = "result_"+method+"_exp"+exp_id+"_proc"+str(proc_label)+".tgz";

exp_info["pid"] = str(os.getpid());
write_config(expid_file, exp_info);

if (method == "treebuild"):
	exp_treebuild();
elif (method == "allpairroute"):
	exp_allpairroute();
elif (method == "allpairrouteght"):
	exp_allpairrouteght();
elif (method == "joinvar"):
	exp_joinvar();
elif (method == "joinmethods"):
	exp_joinmethods();
elif (method == "joinmethodslearn"):
	exp_joinmethods_learn();
elif (method == "joinintel"):
	exp_joinintel();
elif (method == "joinintelstress"):
        exp_joinintelstress();
elif (method == "ghtjoin"):
	exp_ghtjoin();
elif (method == "stresstest"):
	exp_stresstest();
elif (method == "repairest"):
        exp_repairest();
elif (method == "repairsampling"):
	exp_repairsampling();
else:
	print("Invalid method!");
	sys.exit();

system_cmd("ssh "+result_machine+" mkdir -p "+result_dir+"/exp"+exp_id);
system_cmd("scp "+result_filename+" "+user_name+"@"+result_machine+":"+result_dir+"/exp"+exp_id);
